import scrapy
# 导入对应的item对象
from turorial.items import TencentcrawlspiderItem
import json


class TencentSpider(scrapy.Spider):
    name = 'tencent'
    allowed_domains = ['tencent.com']
    # 初始访问页面，直接指定确定的页面
    url = 'https://careers.tencent.com/tencentcareer/api/post/Query?pageSize=10&pageIndex='
    # 设置页码
    page_index = 1
    start_urls = [url+str(page_index)]

    def parse(self, response):
        """
               对下载器返回的响应进行处理的函数
               """
        # 获取当前页面所有的职位信息
        jobs = json.loads(response.text)['Data']['Posts']
        try:
           for job in jobs:
               # 创建一个item对象
               item = TencentcrawlspiderItem()
               # 将item中的各属性进行设置
               item['position_name'] = job['RecruitPostName']
               item['position_addr'] = job['LocationName']
               item['position_type'] = job['CategoryName']
               item['position_detail'] = job['Responsibility']
               item['position_time'] = job['LastUpdateTime']
               yield item

           # 页码加1
           self.page_index += 1

           next_url = self.url + str(self.page_index)
           yield scrapy.Request(next_url, callback=self.parse)
        except TypeError:
            print("爬取结束")


